import base64
import io
import re
import csv
from fontTools.ttLib import TTFont
import requests
from lxml import etree

# 爬取深圳58网页
# url = 'https://sz.58.com/chuzu/'
#
# headers = {
#     'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36',
#     'Cookie': 'id58=c5/nfF+x4NElK0C2JBbHAg==; wmda_uuid=1a8b5e39204217df7b7f0b4966b6f36f; wmda_new_uuid=1; wmda_session_id_11187958619315=1605492949211-e7973df9-1ab3-f654; wmda_visited_projects=%3B11187958619315; 58tj_uuid=10b95ef4-2742-4571-ba5d-4c03367e4b2a; new_session=1; new_uv=1; utm_source=; spm=; init_refer=http%253A%252F%252F19python.howdoit.cn%252F; xxzl_deviceid=74Wzd1W1RDIPn9vX2iIsEVpJCu8i%2B0HAggBeZYs3Ci%2BXyOSNgC3Xbg6URIU0fqx8; als=0'
# }
# r = requests.get(url=url, headers=headers)
# r.encoding = 'utf-8'
# print(r.text)
# g = open('./sz-58.html', 'w')
# g.write(r.text)
f = open('./sz-58.html', 'r')
html_content = f.read()
# 第2步：提取base64 字库数据，解析字库
base64_str = re.search("base64,(.*?)'\)", html_content).group(1)
# print(base64_str)
# 得到了字库的二进制数据
base64_bin = base64.b64decode(base64_str)
# 从字库的二进制数据中转换为字体
font = TTFont(io.BytesIO(base64_bin))
# 从字体中对应关系
cmap = font.get("cmap").getBestCmap()
# print(cmap)
ttf_dict = dict()
for k, v in cmap.items():
    # print(k, v)
    # print(hex(k), v)
    # print(hex(k), int(re.search(r"\d+", v).group(0)))
    new_dict_k = hex(k)
    new_dict_v = int(re.search(r"\d+", v).group(0)) - 1
    ttf_dict[new_dict_k] = new_dict_v
print("解析后：", ttf_dict)
# 第三步：对整个html文档进行替换字库对应的数字
for k, v in ttf_dict.items():
    k = k.replace('0x', '&#x') + ';'
    html_content = html_content.replace(k, str(v))
# 第四步：用xpath提取数据
html = etree.HTML(html_content)
li_list = html.xpath("//li[@class='house-cell']")
# print(li_list)
csv_headers = ['房子介绍', '价格']
g = open('./58sz房价.csv', 'w', encoding='utf-8')
g_csv = csv.writer(g)
g_csv.writerow(csv_headers)
for li in li_list:
    title = li.xpath(".//h2//a/text()")[0].strip()
    price = li.xpath(".//div[@class='money']//b/text()")[0].strip()
    info = [title, price + '元/月']
    print(title, price)
    g_csv.writerow(info)
